'''
    基于0.8677.py修改细节问题，更改了奇怪的一个地方，
    完全正确
    train : 7381 / 2763
    val : 2483 / 918
    test : 2480 / 926
    包含
    train : 7381 / 7327
    val : 2483 / 2452
    test : 2480 / 2457

    current best f1: 0.8677290836653386, epoch: 57 
'''

import spacy
from spacy import displacy
from transformers import BertTokenizer
import json

nlp = spacy.load('en_core_web_trf')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

def select_data (data):
    # 得到包含词性的句子（用于验证提取结果）
    text_path = 'text/text_tag/{}_text_tag.txt'.format(data)
    with open(text_path, 'r', encoding='utf8') as fp:
        text_tag_datas = fp.readlines()

    # 得到无词性的真正的名词短语（用于验证提取结果）
    ner_path = 'ner/ner/{}_ner.json'.format(data)
    with open(ner_path, 'r', encoding='utf8') as fp:
        ner_datas = fp.readlines()

    # 得到无tag且经过预处理的句子（作为提取名词短语的输入）
    text_path = 'text/text/{}_text.json'.format(data)
    with open(text_path, 'r',  encoding='utf8') as fp:
        text_datas = fp.readlines()

    return text_tag_datas, ner_datas, text_datas


def extract_phrase(sentence):

    character = nlp(sentence)
    character_len = len(character)
    # displacy.serve(character, style='dep')

    nn_nps = []         # 存放一个文本的未解决bert分词之前的所有名词短语
    phrase_list = []    # 存放根据nn_nps解决分词之后的所有名词短语，也就是最终返回的
    start_list = []     # 保存当前名词短语中第一个token的下标
    len_list = []       # 保存当前名词短语的长度
    i = 0
    while i < character_len:
        # 提取单引号或双引号内容
        if character[i].text in ['\'', '"', '‘', '“'] and character[i].tag_ in ['\'\'', '``'] and character[
            i - 1].text != ':':
            index = i + 1
            str_temp = ''
            flag = 0
            # 遇到右侧结束符则停止拼接
            while index < character_len and (
                    character[index].text not in ['\'', '"', '‘', '’', '”'] or character[index].tag_ not in ['\'\'',
                                                                                                             '``'] or index + 1 < character_len and character[index + 1].tag_ == 'VBP'):
                if len(str_temp) == 0:
                    str_temp = character[index].text
                    start_id = index
                else:
                    if character[index].text not in ['!']:
                        str_temp = str_temp + ' ' + character[index].text
                index += 1
                flag = 1
            # 判断是否找到右侧结束符
            if index < character_len and flag:  # 判断是否找到右侧结束符
                nn_nps.append(str_temp.rstrip(','))
                start_list.append(start_id)
                i = index + 1
            else:
                i += 1
        elif character[i].tag_ == 'XX' and (
                i - 1 >= 0 and character[i - 1].tag_ in ['NFP', 'UH', '.'] and i == character_len - 1 or
                i - 1 >= 0 and character[i - 1].tag_ in ['IN', '-LRB-', '.'] and i + 1 < character_len and character[
                    i + 1].tag_ in ['IN', '-RRB-', '``'] or \
                i == 0 and character[i + 1].tag_ == 'CC'):
            nn_nps.append(character[i].text)
            start_list.append(i)
            i += 1
        # 提取名词短语中包含PRP$的
        elif character[i].tag_ == 'PRP$' and character[i].text not in ['His']:
            if i - 1 >= 0 and character[i - 1].tag_ in ['UH', 'IN', 'VB'] and character[i - 1].text not in ['for', 'in'] and i + 1 <= len(
                character) and character[i + 1].tag_ in ['NN', 'NNP', 'NNS']:
                if character[i - 1].tag_ == 'VB' and character[i + 1].tag_ == 'NNP':
                    i += 1
                    continue
                nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                start_list.append(i - 1)
                i = i + 2
            elif i + 2 < character_len and character[i + 1].tag_ in ['NNP', 'NNPS'] and \
                    character[i + 2].tag_ not in ['NNP', 'NNPS', '.', 'POS']:
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i = i + 2
            else:
                i += 1
        elif character[i].tag_ == 'ADD' and i - 1 > 0 and character[i - 1].text == '@':
            nn_nps.append(character[i].text)
            start_list.append(i)
            i += 1
        elif character[i].tag_ in ['RB']:
            if i - 1 >= 0 and character[i - 1].tag_ in ['DT'] and character[i].text.isupper() and i + 1 < character_len and character[i + 1].tag_ in ['NNP'] and character[i + 1].text.isupper():
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
            elif i == character_len - 1 and i - 1 >= 0 and character[i - 1].tag_ == '``':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            else:
                i += 1
        elif character[i].tag_ in ['JJ']:
            if i + 2 < character_len and character[i + 1].tag_ == 'JJ' and character[i + 2].text == 'school':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text)
                start_list.append(i)
                i += 3
            elif i + 2 < character_len and character[i + 1].tag_ == 'JJ' and character[i + 2].tag_ != 'NN':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
            elif i - 1 >= 0 and character[i - 1].tag_ == 'DT' and i + 1 < character_len and character[i + 1].tag_ == 'NN':
                nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                start_list.append(i - 1)
                i += 2
            elif i == 0 or i + 2 < character_len and character[i + 1].tag_ == 'NN' and \
                    character[i + 2].tag_ not in ['``', 'IN', 'NNPS', 'VBZ']:
                if character[i + 1].tag_ == 'NN' and character[i + 2].tag_ == ',':
                    nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                    start_list.append(i)
                    i += 2
                else:
                    nn_nps.append(character[i].text)
                    start_list.append(i)
                    i += 1
            elif character[i].text.istitle() and i - 1 >= 0 and character[i - 1].tag_ not in ['IN', '.']:
                if i - 1 >= 0 and character[i - 1].tag_ == 'DT':
                    str_temp = character[i - 1].text + ' ' + character[i].text
                    start_list.append(i - 1)
                else:
                    str_temp = character[i].text
                    start_list.append(i)
                index = i + 1
                while index < character_len and (
                        character[index].text.istitle() and character[index].tag_ not in ['IN'] and
                        index + 1 < character_len and character[index + 1].text not in ['\''] or
                        character[index].text == '-' or character[index].tag_ in ['NNS', 'NN']):
                    str_temp = str_temp + ' ' + character[index].text
                    index += 1
                nn_nps.append(str_temp)
                i = index
            elif character[i].text.istitle():
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            else:
                i += 1
        elif i == 0 and character[i].tag_ in ['CD', 'VBN']:
            if character[i].tag_ == 'CD' and character[i + 1].tag_ != 'NNS':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif character[i].tag_ in ['VBN', 'JJ']:
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            else:
                i += 1
        elif character[i].tag_ in ['NNP', 'NNPS'] and character[i].text not in ['#', '@']:
            # 判断当前词前面的词是否满足拼接条件
            if i - 1 >= 0 and character[i - 1].tag_ in ['PRP$'] and character[i - 1].text not in ['#', '@', 'His'] and \
                    (character[i - 1].head == character[i] or character[i - 1].head == character[i].head):
                str_temp = character[i - 1].text + ' ' + character[i].text
                start_list.append(i - 1)
            elif i - 2 >= 0 and character[i - 2].tag_ in ['DT', 'JJ'] and character[i - 1].tag_ == 'JJ' and character[
                i - 1].text != '#':
                str_temp = character[i - 2].text + ' ' + character[i - 1].text + ' ' + character[i].text
                start_list.append(i - 2)
            elif i - 1 >= 0 and character[i - 1].tag_ in ['VBG', 'CD', 'DT', 'JJ', 'FW'] and character[
                i - 1].text not in ['@', '#']:
                # 前面的词是否被单独取出，如果被单独取出则不拼接
                if len(nn_nps) > 0 and character[i - 1].text in nn_nps:
                    str_temp = character[i].text
                    start_list.append(i)
                else:
                    str_temp = character[i - 1].text + ' ' + character[i].text
                    start_list.append(i - 1)
            elif i - 1 >= 0 and character[i - 1].tag_ in ['NN']:
                if character[i - 1].text in nn_nps:
                    nn_nps.remove(character[i - 1].text)
                    start_list = start_list[0:-1]
                    str_temp = character[i - 1].text + ' ' + character[i].text
                    start_list.append(i - 1)
                else:
                    str_temp = character[i].text
                    start_list.append(i)
            elif i - 2 >= 0 and character[i - 2].tag_ != 'VBN' and i - 1 >= 0 and character[
                i - 1].text == '@' and i + 1 < character_len and character[i + 1].tag_ == 'POS':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
                continue    # 满足此条件表示不需要继续与后面的进行拼接，可退出循环，处理下一个词
            elif i - 1 >= 0 and character[i - 1].tag_ == '.' and i - 2 >= 0 and character[i - 2].tag_ == 'NN':
                str_temp = character[i - 2].text + ' ' + character[i - 1].text + ' ' + character[i].text
                start_list.append(i - 2)
            else:
                str_temp = character[i].text
                start_list.append(i)

            # 判断当前词后面的词是否满足拼接条件
            last = ""
            last_noun = ""
            index = i + 1
            last_token = ''
            flag = 0
            while index < character_len and (
                    character[index].tag_ in ['NNP', 'NNPS', 'NN', 'NNS', 'CD', 'JJ', 'RB', '.', 'POS', 'VBZ', 'VBG',
                                              'VBN', 'JJR', 'PRP', 'PRP$', 'VBP', 'MD', 'VB'] and
                    character[index].text not in ['@', '?', 'out'] and
                    'say' not in character[index].text or character[index].tag_ == 'SYM' and character[index].text in [
                        '@'] or \
                    character[index].text in ['de', 'in', 'In', 'of', 'Of', 'for', 'by', 'on', 'with', 'With', 'to',
                                              'at',  # 对该列表中的元素加规则
                                              'VS', '#', 'and', 'the', 'The', 'THE', 'A', 'N', 'n', 'AND', 'And',
                                              '’', '"', '-', '\'', ':', ',', '+', '/']):
                # 下面if是拼接结束的条件
                if (last_token in ['?', '\'', '-', ',', 'in', 'Firm'] or last_noun in ['POS', 'VBZ', 'DT', 'CD']) and \
                        (character[index].text in ['#', 'of', '\'', 'to', '!', '"', 'a', '-', '"', '#', 'for'] or
                         character[index].tag_ in ['\'', 'IN', 'VBG']):
                    if last_noun == ':' and character[index].tag_ != 'PRP':
                        break
                    elif last_noun != ':':
                        break
                if index + 1 < character_len and character[index].tag_ in ['RB', '.', 'IN'] and \
                        character[index].text != 'VS' and character[index + 1].tag_ in ['JJ', '.', '\'\'']:
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].tag_ == ',' and \
                        character[index - 1].tag_ == 'NNP' and character[index + 1].tag_ in ['JJ']:
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text in ['for', 'to'] and \
                        (character[index - 1].tag_ != 'NNP' or character[index + 1].tag_ != 'NNP'):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == '#' and \
                        (character[index - 1].tag_ not in ['NNP', 'NNPS'] and character[index - 1].text != 'of' or
                         character[index + 1].tag_ not in ['NNP', 'NNPS', 'NN'] and character[
                             index + 1].text != 'festival'):
                    break
                if last_noun == 'FW' and last_token == '#' and index - 2 >= 0 and character[index - 2].tag_ != 'IN':
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text in ['of', 'Of'] and \
                        (character[index - 1].tag_ not in ['NNP', 'NNPS'] and character[index - 1].text != 'mall' or
                         character[index + 1].tag_ not in ['NNP', 'NNPS', 'DT', 'PRP$', 'NNS'] and character[
                             index + 1].text != '#'):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'VS' and \
                        (character[index - 1].tag_ != 'NNP' or character[index + 1].tag_ != '.'):
                    break
                if index - 1 >= 0 and character[index].text == '’' and character[index - 1].text == 'P':
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text in ['for', 'at'] and \
                        (character[index - 1].tag_ != 'NNP' or character[index + 1].tag_ != 'NNP'):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text in ['by'] and \
                        (character[index - 1].tag_ not in ['NNP', 'NNPS'] or character[index + 1].tag_ not in ['NNP', 'DT'] or \
                         character[index - 1].tag_ == 'NNP' and character[index + 1].tag_ == 'NNP' and index + 2 < character_len and character[index + 2].tag_ == '.'):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'on' and \
                        (character[index - 1].text != 'Rock' or character[index + 1].text != 'the'):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text in ['in', 'In'] and \
                        (character[index - 1].tag_ != 'NNP' or character[
                            index - 1].text.istitle() == False or
                         character[index + 1].tag_ not in ['NNP', 'DT']):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'with' and \
                        (character[index - 1].tag_ not in ['NNP', 'VB'] or character[
                            index - 1].text.istitle() == False or
                         character[index + 1].tag_ not in ['NNP', 'DT', 'NN']):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text in ['and'] and \
                        (character[index - 1].tag_ not in ['NNP', 'NNPS'] or character[index + 1].tag_ not in ['NNP', 'NNPS', 'DT', 'JJR']):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'the' and \
                        (character[index - 1].text not in ['of', 'Of', 'in', 'and', 'with', 'on', 'by'] and character[
                            index - 1].tag_ not in ['NNP'] or character[index + 1].tag_ not in ['NNP', 'NNPS', 'VBN']):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'The' and \
                        (character[index - 1].text not in ['In', 'and', ':', 'Of'] or character[index + 1].tag_ not in [
                            'NNP', 'NNPS']):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'THE' and \
                        character[index - 1].text.isupper() == False:
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].text == 'A' and character[
                    index].tag_ == 'DT' and \
                        (character[index - 1].text != ':' or character[index + 1].tag_ != 'JJ'):
                    break
                if index - 1 >= 0 and index + 1 < character_len and character[index].tag_ == 'VBZ' and \
                        (character[index].text not in ['\'', 's', 'styles'] and character[index - 1].tag_ != 'NNP' or
                         character[index + 1].tag_ not in ['NNP', 'JJ', 'RB', 'VBG', 'VBZ', 'IN', 'CC']):
                    break
                if last_token == '/' and index + 1 < character_len and character[index + 1].tag_ == 'CC':
                    break
                if character[index].tag_ == 'PRP$' and (
                        last_noun not in ['CC', 'IN'] or index + 1 < character_len and character[
                    index + 1].tag_ not in ['NN', 'JJ']):
                    break
                if character[index].tag_ == 'VBP' and (
                        last_noun != 'NNP' or index + 1 < character_len and character[index + 1].tag_ != 'IN'):
                    break
                if character[index].text == 'With' and (
                        last_noun != 'VBP' or index + 1 < character_len and character[index + 1].tag_ != 'PRP'):
                    break
                if character[index].text == '-' and last_noun == 'NNP' and index + 1 < character_len and character[
                    index + 1].tag_ == 'VBN':
                    break
                if character[index].tag_ == '``' and (
                        character[i].tag_ == 'NNP' and index + 1 < character_len and character[
                    index + 1].tag_ == 'RB'):
                    flag = 0
                    break
                if character[index].tag_ == 'MD' and (
                        last_noun != ',' or index + 1 < character_len and character[index + 1].tag_ != 'PRP'):
                    break
                if character[index].tag_ == 'PRP' and (
                        last_noun not in ['MD', 'IN', ':'] or index + 1 < character_len and character[
                    index + 1].tag_ not in ['VB', '-LRB-', 'NNPS']):
                    break
                if character[index].tag_ == 'VB' and (
                        last_noun != 'PRP' or index + 1 < character_len and character[index + 1].tag_ != 'IN'):
                    break
                str_temp = str_temp + last  # 拼接的是上个toekn的下一个词
                last = ' ' + character[index].text
                last_token = character[index].text  # 得到当前token
                last_noun = character[index].tag_  # 得到当前token的词性
                index = index + 1
                flag = 1
            # 判断最后一个词是否满足拼接的条件
            if last_noun in ['JJR', 'NNP', 'NNPS', 'NN', 'NNS', 'CD', 'VBZ', 'RB', 'POS', 'VBG', 'JJ',
                             'PRP'] and last_token not in [
                '#', ':', 'of', 'to', '-', '\'', '@', '?', 'signs', 'cancels', 'show', 'posts', 'laughing', 'burns']:
                str_temp = str_temp + last
                i = index
            elif flag == 0:  # 没进去while循环
                i = index
            else:           # 进去while循环，不满足if
                i = index - 1
            nn_nps.append(str_temp)
        elif character[i].tag_ == 'NN' and character[i].text != '@':
            if i == 0:
                np_temp = character[i].text
                start_list.append(i)
                index = i + 1
                while index < character_len and character[index].tag_ in ['NN']:
                    np_temp = np_temp + ' ' + character[index].text
                    index += 1
                i = index
                nn_nps.append(np_temp)
            elif i - 2 >= 0 and character[i - 2].tag_ == 'JJ' and character[i - 1].tag_ == 'JJ':
                str_temp = character[i - 2].text + ' ' + character[i - 1].text
                # JJ JJ未被取出，则进行提取
                if str_temp not in nn_nps:
                    str_temp = str_temp + ' ' + character[i].text
                    nn_nps.append(str_temp)
                    start_list.append(i - 2)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ in ['JJ', 'VBN', 'CD', 'PRP'] and character[i - 1].text not in [
                'ne1', '#']:
                if i + 1 < character_len and character[i + 1].tag_ in ['NN']:
                    # 如果前一个词被提取出，则先将其从列表中移除
                    if character[i - 1].text in nn_nps:
                        nn_nps.remove(character[i - 1].text)
                        start_list.remove(i - 1)
                    # 拼接
                    nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                    start_list.append(i - 1)
                    i += 2
                elif i + 1 < character_len and character[i + 1].tag_ in [':']:
                    if character[i - 1].text in nn_nps:
                        nn_nps.remove(character[i - 1].text)
                        start_list.remove(i - 1)
                    nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                    start_list.append(i - 1)
                    i += 1
                # 如果当前列表不为空，并且列表最后一个元素不是当前词的上一个词
                elif len(nn_nps) != 0 and nn_nps[-1] != character[i - 1].text:
                    if i + 1 < character_len and character[i + 1].tag_ == 'NNPS':
                        nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                        start_list.append(i - 1)
                        i += 2
                    else:
                        nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                        start_list.append(i - 1)
                        i += 1
                elif character[i - 1].tag_ == 'CD' and i + 1 < character_len and character[i + 1].tag_ == ',':
                    nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                    start_list.append(i - 1)
                    i += 1
                elif i + 3 < character_len and character[i + 1].tag_ == character[i + 2].tag_ == 'POS' and character[i + 3].tag_ == 'NNP':
                    nn_nps.append(
                        character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text + ' ' + character[
                            i + 3].text)
                    start_list.append(i)
                    i += 4
                elif character[i - 1].tag_ == 'JJ' and i + 1 < character_len and character[i + 1].tag_ == '.':
                    nn_nps.append(character[i].text)
                    start_list.append(i)
                    i += 1
                else:
                    i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ in ['IN'] and i + 2 < character_len and character[
                i + 1].tag_ in ['NN'] and character[i + 2].tag_ in ['VBZ']:
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
            elif i - 1 >= 0 and character[i - 1].tag_ in ['NN', 'POS', 'IN'] and i + 1 < character_len and character[
                i + 1].tag_ in ['NN', ':']:
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ in ['IN'] and i == character_len - 1:
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif character[i - 1].tag_ == 'IN' and i + 2 < character_len and character[
                i + 1].tag_ == 'CD' and character[i + 2].tag_ == 'IN':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
            elif i - 1 >= 0 and character[i - 1].text in ['@'] and i + 2 < character_len:
                nn_nps.append(character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text)
                start_list.append(i)
                i += 3
            elif i - 1 >= 0 and character[i - 1].tag_ in ['WRB', 'VBZ', 'VB', 'VBN', 'VBD', 'VBG', 'JJS', 'JJR', 'CC',
                                                          'PRP$', ':'] or character[i - 1].text in ['@', '#']:
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ == 'DT' and i + 1 < character_len and character[
                i + 1].tag_ == 'NN':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ in ['IN', 'VB', 'DT'] and character[i - 1].text not in ['on',
                                                                                                              'this',
                                                                                                              'This']:
                if i + 4 < character_len and character[i + 1].tag_ in ['IN', 'NNP'] and \
                        character[i + 2].tag_ in ['NNP', 'CC'] and \
                        i + 4 < character_len and character[i + 4].tag_ != '.':
                    i += 1
                    continue
                np_temp = character[i].text
                index = i + 1
                last_noun = ''
                while index < character_len and (
                        character[index].tag_ in ['NN', 'NNP', 'PRP', 'IN'] or character[index].text in ['.']):
                    if index + 1 < character_len and character[index].text in ['.'] and character[index + 1].tag_ in [
                        '``']:
                        index += 1
                        break
                    last_noun = character[index].tag_
                    np_temp = np_temp + ' ' + character[index].text
                    index += 1
                if last_noun != 'IN':
                    nn_nps.append(np_temp)
                    start_list.append(i)
                    i = index
                else:
                    i += 1
            elif i + 1 < character_len and character[i + 1].tag_ == 'VBZ':
                nn_nps.append(character[i + 1].text)
                start_list.append(i + 1)
                i += 2
            elif i - 1 >= 0 and character[i - 1].tag_ == 'DT' and i + 1 < character_len and character[
                i + 1].tag_ == 'NN':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ == 'NN' and character[i - 1].text.isupper() and character[
                i].text.isupper() and i + 1 < character_len and character[i + 1].tag_ == 'NNP' and character[
                i + 1].text.isupper():
                if character[i - 1].text in nn_nps:
                    nn_nps.remove(character[i - 1].text)
                    start_list.remove(i - 1)
                nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                start_list.append(i - 1)
                i += 2
            else:
                i += 1
        elif character[i].tag_ == 'NNS':
            if i == 0:
                np_temp = character[i].text
                index = i + 1
                while index < character_len and character[index].tag_ == 'NN':
                    np_temp = np_temp + ' ' + character[index].text
                    index += 1
                nn_nps.append(np_temp)
                start_list.append(i)
                i = index
            elif i + 1 < character_len and character[i + 1].tag_ in ['RB', 'JJ']:
                if i - 1 >= 0 and character[i - 1].tag_ == 'CD':
                    nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                    start_list.append(i - 1)
                    i += 2
                else:
                    nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                    start_list.append(i)
                    i += 2
            elif i - 1 >= 0 and character[i - 1].tag_ in ['VB', 'VBP', 'IN', 'WRB', 'DT', 'PRP$', 'VBG'] \
                    or character[i - 1].text in ['@', '#', ':', '!']:
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ in ['CD', 'JJ']:
                if i - 2 >= 0 and character[i - 2].tag_ == 'JJ' and character[i - 1].tag_ == 'JJ':  # 需要放在前面
                    nn_nps.append(character[i - 2].text + ' ' + character[i - 1].text + ' ' + character[i].text)
                    start_list.append(i - 2)
                    i += 1
                elif character[i - 1].tag_ == 'JJ' and i + 1 < character_len and character[i + 1].tag_ == ':':
                    if character[i - 1].text in nn_nps:
                        nn_nps.remove(character[i - 1].text)
                        start_list.remove(i - 1)
                    nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                    start_list.append(i - 1)
                    i += 1
                elif len(nn_nps) > 0 and nn_nps[-1] != character[i - 1].text and character[i - 1].tag_ == 'JJ' or \
                        character[i - 1].tag_ == 'CD':
                    start_list.append(i - 1)
                    if character[i - 1].tag_ == 'JJ' and i + 1 < character_len and character[i + 1].tag_ == 'NN':
                        nn_nps.append(character[i - 1].text + ' ' + character[i].text + ' ' + character[i + 1].text)
                        i += 2
                    elif character[i - 1].tag_ == 'CD' or i + 1 < character_len and character[i + 1].tag_ != 'NN':
                        nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                        i += 1
                    else:
                        i += 1
                else:
                    i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ == 'NN' and character[i - 1].text not in ['baseball']:
                nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                start_list.append(i - 1)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ == '``' and i + 1 >= 0 and character[i + 1].tag_ == 'MD':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            else:
                i += 1
        elif character[i].tag_ == 'VB':
            # 处理不需要进入while循环进行拼接的情况
            if i + 4 < character_len and character[i + 1].tag_ == 'NNP' and character[i + 2].tag_ == 'IN' and \
                    character[i + 3].tag_ == 'NNPS' and character[i + 4].tag_ == 'JJ':
                nn_nps.append(character[i + 1].text + ' ' + character[i + 2].text + ' ' + character[i + 3].text)
                start_list.append(i + 1)
                i += 4
                continue
            elif i - 1 >= 0 and character[i - 1].tag_ == 'TO' and i + 2 < character_len and character[
                i + 1].tag_ == 'NNP' and character[i + 2].tag_ != 'CC':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
                continue
            elif i - 1 >= 0 and character[i - 1].tag_ == 'TO' and i + 1 < character_len and character[
                i + 1].tag_ == 'IN':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
                continue
            elif i == 0 and i + 1 >= 0 and character[i + 1].tag_ == 'DT' and i + 2 >= 0 and character[
                i + 2].tag_ == 'NN':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text)
                start_list.append(i)
                i += 3
                continue
            elif i - 1 >= 0 and character[i - 1].tag_ in ['TO', '.', 'MD']:
                i += 1
                continue
            elif i - 1 >= 0 and character[i - 1].tag_ == 'VBD':
                np_temp = character[i - 1].text + ' ' + character[i].text
                star_id = i - 2
            else:
                np_temp = character[i].text
                star_id = i
            index = i + 1
            flag = 0
            if i == 0 and character[i + 1].head != character[i]:
                i += 1
                continue
            while index < character_len and (
                    character[index].tag_ in ['PRP$', 'WP'] or character[index].tag_ in ['RP', 'WP', 'PRP', 'NN', 'NNP',
                                                                                         'JJ']
                    or character[index].text in ['For'] and character[index].head == character[i]):
                np_temp = np_temp + ' ' + character[index].text
                index += 1
                flag = 1
            if flag:  # 防止单独提取出VB
                nn_nps.append(np_temp)
                start_list.append(star_id)
            i = index
        elif character[i].tag_ == 'VBZ' and i + 1 < character_len:
            if i == 1 and character[i - 1].tag_ == 'NNP':
                if character[i - 1].text in nn_nps:
                    nn_nps.remove(character[i - 1].text)
                    start_list.remove(i - 1)
                nn_nps.append(character[i - 1].text + ' ' + character[i].text)
                start_list.append(i - 1)
                i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ == 'CC' and i + 1 < character_len and character[
                i + 1].tag_ == 'NNP':
                start_list.append(i)
                nn_nps.append(character[i].text)
                i += 1
            elif character[i + 1].tag_ == ':':
                start_list.append(i)
                nn_nps.append(character[i].text)
                i += 1
            elif character[i].text not in ['is']:
                str_temp = character[i + 1].text
                flag = 0
                index = i + 2
                while index < character_len:
                    if character[index].head == character[i] and character[index].tag_ == 'IN':
                        flag = 1
                        break
                    str_temp += ' ' + character[index].text
                    index += 1
                if flag and len(str_temp.split()) > 1:
                    start_list.append(i + 1)
                    nn_nps.append(str_temp)
                    i = index
                else:
                    i += 1
            else:
                i += 1
        elif character[i].tag_ == 'CD' and i + 1 < character_len and (character[i + 1].tag_ in ['CD', 'FW'] and \
                                                                       character[i].head == character[i + 1] or
                                                                       character[i + 1].tag_ in ['UH']):
            nn_nps.append(character[i].text + ' ' + character[i + 1].text)
            start_list.append(i)
            i += 2
        elif character[i].tag_ == 'FW' and i - 1 > 0 and character[i - 1].text == '#' and \
                (i == character_len - 1 or i + 1 < character_len and (
                        character[i + 1].text in ['#', ',', '…'] or character[i + 1].tag_ == 'NN')):
            nn_nps.append(character[i].text)
            start_list.append(i)
            i += 1
        elif character[i].tag_ == 'DT':
            if i + 1 < character_len and character[i + 1].tag_ == 'WP':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
            elif i - 1 >= 0 and character[i - 1].tag_ == 'NFP' and i + 3 < character_len and \
                    character[i + 1].tag_ == 'MD' and character[i + 2].tag_ == 'VB' and character[i + 3].tag_ == 'PRP':
                nn_nps.append(
                    character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text + ' ' + character[
                        i + 3].text)
                start_list.append(i)
                i += 4
            else:
                i += 1
        elif character[i].tag_ == 'PRP':
            if i == 0:
                if character[i + 1].tag_ == 'VBD' and character[i + 2].tag_ == 'VBG' and character[i + 3].tag_ == 'IN':
                    nn_nps.append(character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text)
                    start_list.append(i)
                    i += 3
                    continue
                str_temp = character[i].text
                index = i + 1
                last_tag = ''
                while index < character_len and character[index].tag_ in ['PRP', '.', 'VBP']:
                    str_temp = str_temp + ' ' + character[index].text
                    last_tag = character[index].tag_
                    index += 1
                list_temp = str_temp.split()
                len_temp = len(list_temp)
                if len_temp == 6 and last_tag == '.':
                    nn_nps.append(' '.join(list_temp[0:len_temp - 1]))
                    start_list.append(i)
                    i = index + 1
                else:
                    i = index
            elif i == character_len - 1 and i - 1 >= 0 and character[i - 1].tag_ == 'NFP':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i + 3 < character_len and \
                    character[i + 1].tag_ == 'VBP' and character[i + 2].tag_ == 'IN' and character[i + 3].tag_ == 'PRP':
                nn_nps.append(
                    character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text + ' ' + character[
                        i + 3].text)
                start_list.append(i)
                i += 4
            elif i - 1 >= 0 and character[i - 1].tag_ == ':':
                str_temp = character[i].text
                star_id = i
                index = i + 1
                str_temp_len = 1
                while index < character_len and character[index].tag_ != '-LRB-':
                    str_temp += ' ' + character[index].text
                    str_temp_len += 1
                    index += 1
                if str_temp_len == 8 and index < character_len and character[index].tag_ == '-LRB-':
                    nn_nps.append(str_temp)
                    start_list.append(star_id)
                    i += 8
                if str_temp_len == 5 and index == character_len:
                    nn_nps.append(str_temp)
                    start_list.append(star_id)
                    i += 5
                else:
                    i += 1
            elif i - 1 >= 0 and character[i - 1].tag_ == 'VB' and i + 1 < character_len and character[
                i + 1].tag_ == 'VBD':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif i - 2 >= 0 and character[i - 2].tag_ == ':' and i - 1 >= 0 and character[
                i - 1].tag_ == 'WP' and i + 1 < character_len and character[i + 1].tag_ == 'VBP':
                str_temp = character[i - 1].text
                star_id = i - 1
                index = i
                str_temp_len = 1
                while index < character_len:
                    str_temp += ' ' + character[index].text
                    str_temp_len += 1
                    index += 1
                if str_temp_len == 12:
                    nn_nps.append(str_temp)
                    start_list.append(star_id)
                    break
                else:
                    i += 1
            else:
                i += 1
        elif character[i].tag_ == 'VBN' and i - 1 > 0 and character[i - 1].tag_ in ['IN', ':'] and i + 1 < character_len:
            if character[i + 1].tag_ == ',':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif character[i + 1].tag_ == 'DT' and i + 2 < character_len and character[i + 2].tag_ == 'NN':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text + ' ' + character[i + 2].text)
                start_list.append(i)
                i += 3
            elif character[i + 1].tag_ == 'NN' and i + 2 < character_len and character[i + 2].tag_ == '``':
                nn_nps.append(character[i].text + ' ' + character[i + 1].text)
                start_list.append(i)
                i += 2
            else:
                i += 1
        elif character[i].tag_ == 'VBG' and i - 1 >= 0 and character[i - 1].tag_ in ['IN', 'NFP'] and i + 1 < character_len and character[i + 1].tag_ in ['VBG', 'IN']:
            nn_nps.append(character[i].text)
            start_list.append(i)
            i += 1
        elif character[i].tag_ == 'TO' and i - 1 > 0 and character[i - 1].tag_ == 'RP' and \
                i + 1 < character_len and character[i + 1].tag_ == 'TO':
            nn_nps.append(character[i].text)
            start_list.append(i)
            i += 1
        elif character[i].tag_ == 'UH' and (
                i == character_len - 1 and character[i - 1].tag_ == 'UH' or i + 1 < character_len and character[
            i + 1].tag_ == ','):
            nn_nps.append(character[i].text)
            start_list.append(i)
            i += 1
        elif character[i].tag_ == 'IN':
            if i - 1 > 0 and character[i - 1].tag_ == 'IN' and i + 1 < character_len and character[i + 1].tag_ == '.':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            elif character[i].text.istitle() and i == character_len - 1 and character[i - 1].tag_ == ',':
                nn_nps.append(character[i].text)
                start_list.append(i)
                i += 1
            else:
                i += 1
        else:
            i = i + 1

    # # 5.遍历nn_nps，计算每个名词短语的长度length，并解决bert分词问题
    # for np in nn_nps:
    #     length = 0
    #     for word in np.split(" "):  # 遍历名词短语中的每个词，进行分词
    #         word_tokens = tokenizer._tokenize(word)  # 得到分词结果的列表
    #         # 如果分词属于那几种特殊情况，则使用第一个分词结果np_tokens[0]替换名词短语中的word
    #         if len(word_tokens) > 1 and ("_" in word or "." in word or "," in word or "##" not in word_tokens[1]):  # 解决bert分词问题
    #             length += 1
    #             np = np.replace(word, word_tokens[0])
    #         # 否则，如果分词正常，则根据分词结果累计名词短语总长度
    #         else:
    #             length += len(word_tokens)
    #
    #     phrase_list.append(np)
    #     len_list.append(length)

    return [nn_nps, start_list, len_list]

if __name__ == '__main__':

    for data in ['train', 'val', 'test']:
        text_tag_datas, ner_datas, text_datas = select_data(data)
        ners_len, inclu_correct = 0, 0
        with open('{}_error.txt'.format(data), 'a', encoding='utf8') as f3:
            for i in range(0, len(text_datas)):
                sentence = json.loads(text_datas[i])
                ners = json.loads(ner_datas[i])
                ners_len += len(ners)           # 真正名词短语总个数

                phrases = extract_phrase(sentence)[0]
                # ext_correct += len(list(set(ners) & set(phrases)))       # 提取出的完全正确的个数
                # 输出跨界或未提取出的例子
                for ner in ners:
                    for phrase in phrases:
                        if ner in phrase:
                            inclu_correct += 1    
                            break 
            print(f"{data} : {ners_len} / {inclu_correct}")

